{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install statsmodels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorboard.backend.event_processing.event_accumulator import EventAccumulator\n",
    "import glob\n",
    "import os\n",
    "import fnmatch\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from scipy import interpolate\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib as mplt\n",
    "import scipy.stats as st\n",
    "import statsmodels.stats.api as sms\n",
    "import colorsys\n",
    "from matplotlib.ticker import (MultipleLocator, FormatStrFormatter)\n",
    "import math\n",
    "\n",
    "# CIFAR100, scaling crossvalidation \n",
    "\n",
    "ROOT_DIR='/mnt/home/boris/experiments_task_encoder/180423_132821_mini_imagenet_metric_multiplier_trainable_encoder_classifier_link_feat_extract_pretrain_metric_multiplier_init_number_of_steps_cbn_per_block_cbn_num_layers_repeat_metric_scaling_crossvalidation_CIFAR100'\n",
    "x_axis_vals = [0.5, 1, 5, 10, 20]\n",
    "number_of_steps = 10000\n",
    "feat_extract_pretrain='multitask'  # None,'multitask'\n",
    "encoder_classifier_link='cbn'\n",
    "experiement_patterns = [\n",
    "    \n",
    "    '*feat_extract_pretrain=%s;encoder_classifier_link=%s;number_of_steps=%d;*metric_multiplier_init=0.5' \n",
    "    %(feat_extract_pretrain,encoder_classifier_link,number_of_steps),\n",
    "    '*feat_extract_pretrain=%s;encoder_classifier_link=%s;number_of_steps=%d;*metric_multiplier_init=1'\n",
    "    %(feat_extract_pretrain,encoder_classifier_link,number_of_steps),\n",
    "    '*feat_extract_pretrain=%s;encoder_classifier_link=%s;number_of_steps=%d;*metric_multiplier_init=5'\n",
    "    %(feat_extract_pretrain,encoder_classifier_link,number_of_steps),\n",
    "    '*feat_extract_pretrain=%s;encoder_classifier_link=%s;number_of_steps=%d;*metric_multiplier_init=10'\n",
    "    %(feat_extract_pretrain,encoder_classifier_link,number_of_steps),\n",
    "    '*feat_extract_pretrain=%s;encoder_classifier_link=%s;number_of_steps=%d;*metric_multiplier_init=20'\n",
    "    %(feat_extract_pretrain,encoder_classifier_link,number_of_steps),\n",
    "]\n",
    "\n",
    "# MINIIMAGENET, 5-shot\n",
    "# This is the first result\n",
    "ROOT_DIR='/mnt/home/boris/experiments_task_encoder/180422_013228_mini_imagenet_metric_multiplier_trainable_encoder_classifier_link_feat_extract_pretrain_polynomial_metric_order_metric_multiplier_init_cbn_per_block_cbn_num_layers_repeat_scaling_crossvalidation_min'\n",
    "# Scaling paramter crossvalidation, polynomial head, TBN, WITH multitask\n",
    "x_axis_vals = [0.5, 1, 5, 10, 20]\n",
    "experiement_patterns = [\n",
    "    '*cbn_num_layers=3*pretrain=multitask*encoder_classifier_link=cbn*cbn_per_block=False*multiplier_init=0.5',\n",
    "    '*cbn_num_layers=3*pretrain=multitask*encoder_classifier_link=cbn*cbn_per_block=False*multiplier_init=1',\n",
    "    '*cbn_num_layers=3*pretrain=multitask*encoder_classifier_link=cbn*cbn_per_block=False*multiplier_init=5',\n",
    "    '*cbn_num_layers=3*pretrain=multitask*encoder_classifier_link=cbn*cbn_per_block=False*multiplier_init=10',\n",
    "    '*cbn_num_layers=3*pretrain=multitask*encoder_classifier_link=cbn*cbn_per_block=False*multiplier_init=20',\n",
    "]\n",
    "# This is a bit better result with the last maxpool layer before avg pool removed\n",
    "ROOT_DIR='/mnt/home/boris/experiments_task_encoder/180508_195508_mini_imagenet_num_max_pools_encoder_classifier_link_feat_extract_pretrain_metric_multiplier_init_cbn_num_layers_repeat_scale_maxpool3'\n",
    "x_axis_vals = [0.5, 1, 5, 7.5, 10, 20]\n",
    "experiement_patterns = [\n",
    "    '*cbn_num_layers=3*pretrain=multitask*encoder_classifier_link=cbn*multiplier_init=0.5',\n",
    "    '*cbn_num_layers=3*pretrain=multitask*encoder_classifier_link=cbn*multiplier_init=1',\n",
    "    '*cbn_num_layers=3*pretrain=multitask*encoder_classifier_link=cbn*multiplier_init=5',\n",
    "    '*cbn_num_layers=3*pretrain=multitask*encoder_classifier_link=cbn*multiplier_init=7.5',\n",
    "    '*cbn_num_layers=3*pretrain=multitask*encoder_classifier_link=cbn*multiplier_init=10',\n",
    "    '*cbn_num_layers=3*pretrain=multitask*encoder_classifier_link=cbn*multiplier_init=20',\n",
    "]\n",
    "# # Scaling paramter crossvalidation, polynomial head, TBN, WITHOUT multitask\n",
    "# experiement_patterns = [\n",
    "#     '*cbn_num_layers=3*pretrain=None*encoder_classifier_link=cbn*cbn_per_block=False*multiplier_init=0.5',\n",
    "#     '*cbn_num_layers=3*pretrain=None*encoder_classifier_link=cbn*cbn_per_block=False*multiplier_init=1',\n",
    "#     '*cbn_num_layers=3*pretrain=None*encoder_classifier_link=cbn*cbn_per_block=False*multiplier_init=5',\n",
    "#     '*cbn_num_layers=3*pretrain=None*encoder_classifier_link=cbn*cbn_per_block=False*multiplier_init=10',\n",
    "#     '*cbn_num_layers=3*pretrain=None*encoder_classifier_link=cbn*cbn_per_block=False*multiplier_init=20',\n",
    "# ]\n",
    "# # The scaling parameter crossvalidation for polynomial head, WITHOUT multitask, (two times more steps!)\n",
    "# experiement_patterns = [\n",
    "#     '*pretrain=None*encoder_classifier_link=polynomial*number_of_steps=60000*multiplier_init=0.5',\n",
    "#     '*pretrain=None*encoder_classifier_link=polynomial*number_of_steps=60000*multiplier_init=1',\n",
    "#     '*pretrain=None*encoder_classifier_link=polynomial*number_of_steps=60000*multiplier_init=5',\n",
    "#     '*pretrain=None*encoder_classifier_link=polynomial*number_of_steps=60000*multiplier_init=10',\n",
    "#     '*pretrain=None*encoder_classifier_link=polynomial*number_of_steps=60000*multiplier_init=20',\n",
    "# ]\n",
    "# # The scaling parameter crossvalidation for polynomial head, WITH multitask, 30000 steps\n",
    "# experiement_patterns = [\n",
    "#     '*pretrain=multitask*encoder_classifier_link=polynomial*multiplier_init=0.5',\n",
    "#     '*pretrain=multitask*encoder_classifier_link=polynomial*multiplier_init=1',\n",
    "#     '*pretrain=multitask*encoder_classifier_link=polynomial*multiplier_init=5',\n",
    "#     '*pretrain=multitask*encoder_classifier_link=polynomial*multiplier_init=10',\n",
    "#     '*pretrain=multitask*encoder_classifier_link=polynomial*multiplier_init=20',\n",
    "# ]\n",
    "\n",
    "\n",
    "\n",
    "# ### 1-shot results\n",
    "# # MINIIMAGENET\n",
    "# ROOT_DIR='/mnt/home/boris/experiments_task_encoder/180503_102004_mini_imagenet_num_tasks_per_batch_encoder_classifier_link_feat_extract_pretrain_train_batch_size_metric_multiplier_init_number_of_steps_cbn_num_layers_repeat_scale_1shot_first_run'\n",
    "# x_axis_vals = [0.2, 0.5, 1, 2.5, 5]\n",
    "# cbn_num_layers=3\n",
    "# number_of_steps=40000\n",
    "# encoder_classifier_link='polynomial'  # 'polynomial', 'cbn'\n",
    "# feat_extract_pretrain='None'  # None 'multitask'\n",
    "# experiement_patterns = [\n",
    "#     '*cbn_num_layers=%d*feat_extract_pretrain=%s;encoder_classifier_link=%s;number_of_steps=%d*metric_multiplier_init=0.2' \n",
    "#     %(cbn_num_layers, feat_extract_pretrain, encoder_classifier_link, number_of_steps),\n",
    "#     '*cbn_num_layers=%d*feat_extract_pretrain=%s;encoder_classifier_link=%s;number_of_steps=%d*metric_multiplier_init=0.5' \n",
    "#     %(cbn_num_layers, feat_extract_pretrain, encoder_classifier_link, number_of_steps),\n",
    "#     '*cbn_num_layers=%d*feat_extract_pretrain=%s;encoder_classifier_link=%s;number_of_steps=%d*metric_multiplier_init=1' \n",
    "#     %(cbn_num_layers, feat_extract_pretrain, encoder_classifier_link, number_of_steps),\n",
    "#     '*cbn_num_layers=%d*feat_extract_pretrain=%s;encoder_classifier_link=%s;number_of_steps=%d*metric_multiplier_init=2.5' \n",
    "#     %(cbn_num_layers, feat_extract_pretrain, encoder_classifier_link, number_of_steps),\n",
    "#     '*cbn_num_layers=%d*feat_extract_pretrain=%s;encoder_classifier_link=%s;number_of_steps=%d*metric_multiplier_init=5' \n",
    "#     %(cbn_num_layers, feat_extract_pretrain, encoder_classifier_link, number_of_steps),\n",
    "# ]\n",
    "\n",
    "\n",
    "# ### 1-shot results\n",
    "# # CIFAR-100\n",
    "# ROOT_DIR='/mnt/home/boris/experiments_task_encoder/180505_022014_mini_imagenet_num_tasks_per_batch_encoder_classifier_link_feat_extract_pretrain_train_batch_size_metric_multiplier_init_number_of_steps_cbn_num_layers_repeat_scale_crossvalidation_CIFAR100_1shot'\n",
    "# x_axis_vals = [0.5, 1, 1.5, 2, 2.5]\n",
    "# cbn_num_layers=3\n",
    "# number_of_steps=10000\n",
    "# encoder_classifier_link='polynomial'  # 'polynomial', 'cbn'\n",
    "# feat_extract_pretrain='None'  # None 'multitask'\n",
    "# experiement_patterns = [\n",
    "#     '*cbn_num_layers=%d*feat_extract_pretrain=%s;encoder_classifier_link=%s;number_of_steps=%d*metric_multiplier_init=0.5' \n",
    "#     %(cbn_num_layers, feat_extract_pretrain, encoder_classifier_link, number_of_steps),\n",
    "#     '*cbn_num_layers=%d*feat_extract_pretrain=%s;encoder_classifier_link=%s;number_of_steps=%d*metric_multiplier_init=1' \n",
    "#     %(cbn_num_layers, feat_extract_pretrain, encoder_classifier_link, number_of_steps),\n",
    "#     '*cbn_num_layers=%d*feat_extract_pretrain=%s;encoder_classifier_link=%s;number_of_steps=%d*metric_multiplier_init=1.5' \n",
    "#     %(cbn_num_layers, feat_extract_pretrain, encoder_classifier_link, number_of_steps),\n",
    "#     '*cbn_num_layers=%d*feat_extract_pretrain=%s;encoder_classifier_link=%s;number_of_steps=%d*metric_multiplier_init=2' \n",
    "#     %(cbn_num_layers, feat_extract_pretrain, encoder_classifier_link, number_of_steps),\n",
    "#     '*cbn_num_layers=%d*feat_extract_pretrain=%s;encoder_classifier_link=%s;number_of_steps=%d*metric_multiplier_init=2.5' \n",
    "#     %(cbn_num_layers, feat_extract_pretrain, encoder_classifier_link, number_of_steps),\n",
    "# ]\n",
    "\n",
    "\n",
    "# ### 10-shot results\n",
    "# # MINIIMAGENET\n",
    "# ROOT_DIR='/mnt/home/boris/experiments_task_encoder/180506_020918_mini_imagenet_encoder_classifier_link_feat_extract_pretrain_metric_multiplier_init_init_learning_rate_number_of_steps_cbn_num_layers_repeat_scale_10shot'\n",
    "# x_axis_vals = [0.5, 1, 5, 10, 15, 20]\n",
    "# cbn_num_layers=3\n",
    "# number_of_steps=60000\n",
    "# encoder_classifier_link='polynomial'  # 'polynomial', 'cbn'\n",
    "# feat_extract_pretrain='None'  # None 'multitask'\n",
    "# experiement_patterns = [\n",
    "#     '*cbn_num_layers=%d*feat_extract_pretrain=%s;encoder_classifier_link=%s;number_of_steps=%d*metric_multiplier_init=0.5' \n",
    "#     %(cbn_num_layers, feat_extract_pretrain, encoder_classifier_link, number_of_steps),\n",
    "#     '*cbn_num_layers=%d*feat_extract_pretrain=%s;encoder_classifier_link=%s;number_of_steps=%d*metric_multiplier_init=1' \n",
    "#     %(cbn_num_layers, feat_extract_pretrain, encoder_classifier_link, number_of_steps),\n",
    "#     '*cbn_num_layers=%d*feat_extract_pretrain=%s;encoder_classifier_link=%s;number_of_steps=%d*metric_multiplier_init=5' \n",
    "#     %(cbn_num_layers, feat_extract_pretrain, encoder_classifier_link, number_of_steps),\n",
    "#     '*cbn_num_layers=%d*feat_extract_pretrain=%s;encoder_classifier_link=%s;number_of_steps=%d*metric_multiplier_init=10' \n",
    "#     %(cbn_num_layers, feat_extract_pretrain, encoder_classifier_link, number_of_steps),\n",
    "#     '*cbn_num_layers=%d*feat_extract_pretrain=%s;encoder_classifier_link=%s;number_of_steps=%d*metric_multiplier_init=15' \n",
    "#     %(cbn_num_layers, feat_extract_pretrain, encoder_classifier_link, number_of_steps),\n",
    "#     '*cbn_num_layers=%d*feat_extract_pretrain=%s;encoder_classifier_link=%s;number_of_steps=%d*metric_multiplier_init=20' \n",
    "#     %(cbn_num_layers, feat_extract_pretrain, encoder_classifier_link, number_of_steps),\n",
    "# ]\n",
    "\n",
    "# ### 10-shot results\n",
    "# # CIFAR100\n",
    "# ROOT_DIR='/mnt/home/boris/experiments_task_encoder/180507_015250_mini_imagenet_num_tasks_per_batch_encoder_classifier_link_feat_extract_pretrain_metric_multiplier_init_init_learning_rate_number_of_steps_cbn_num_layers_repeat_scale_crossval_CIFAR100_10shot'\n",
    "# x_axis_vals = [0.5, 1, 5, 10, 15, 20]\n",
    "# cbn_num_layers=3\n",
    "# number_of_steps=20000\n",
    "# encoder_classifier_link='polynomial'  # 'polynomial', 'cbn'\n",
    "# feat_extract_pretrain='None'  # None 'multitask'\n",
    "# experiement_patterns = [\n",
    "#     '*cbn_num_layers=%d*feat_extract_pretrain=%s;encoder_classifier_link=%s;number_of_steps=%d*metric_multiplier_init=0.5' \n",
    "#     %(cbn_num_layers, feat_extract_pretrain, encoder_classifier_link, number_of_steps),\n",
    "#     '*cbn_num_layers=%d*feat_extract_pretrain=%s;encoder_classifier_link=%s;number_of_steps=%d*metric_multiplier_init=1' \n",
    "#     %(cbn_num_layers, feat_extract_pretrain, encoder_classifier_link, number_of_steps),\n",
    "#     '*cbn_num_layers=%d*feat_extract_pretrain=%s;encoder_classifier_link=%s;number_of_steps=%d*metric_multiplier_init=5' \n",
    "#     %(cbn_num_layers, feat_extract_pretrain, encoder_classifier_link, number_of_steps),\n",
    "#     '*cbn_num_layers=%d*feat_extract_pretrain=%s;encoder_classifier_link=%s;number_of_steps=%d*metric_multiplier_init=10' \n",
    "#     %(cbn_num_layers, feat_extract_pretrain, encoder_classifier_link, number_of_steps),\n",
    "#     '*cbn_num_layers=%d*feat_extract_pretrain=%s;encoder_classifier_link=%s;number_of_steps=%d*metric_multiplier_init=15' \n",
    "#     %(cbn_num_layers, feat_extract_pretrain, encoder_classifier_link, number_of_steps),\n",
    "#     '*cbn_num_layers=%d*feat_extract_pretrain=%s;encoder_classifier_link=%s;number_of_steps=%d*metric_multiplier_init=20' \n",
    "#     %(cbn_num_layers, feat_extract_pretrain, encoder_classifier_link, number_of_steps),\n",
    "# ]\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf; print(tf.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get directories for the  experiements "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_experiment_folder_list(experiement_dir, experiment_pattern):\n",
    "    list_experiemnts_pattern = []\n",
    "    list_experiments_all = os.listdir(experiement_dir)\n",
    "    for folder in list_experiments_all:\n",
    "        names = os.path.join(ROOT_DIR,folder)\n",
    "#         print(names)\n",
    "        names_filt_alone = fnmatch.filter([names], experiment_pattern)\n",
    "#         print(names_filt_alone)\n",
    "        list_experiemnts_pattern.extend(names_filt_alone)\n",
    "    return list_experiemnts_pattern\n",
    "\n",
    "\n",
    "def interpolate_data(experiment_data, index_max, step):\n",
    "    index_max = int(round(index_max/step)*step)\n",
    "    index_new = np.arange(0, index_max+step, step)\n",
    "    experiment_data_interpolated = {}\n",
    "    for file, df in experiment_data.items():\n",
    "        interpolator = interpolate.interp1d(df.index, df.value, bounds_error=False, fill_value=np.nan)\n",
    "        value_new = interpolator(index_new)\n",
    "        experiment_data_interpolated[file] = value_new\n",
    "    \n",
    "    return pd.DataFrame(index=index_new, data=experiment_data_interpolated)\n",
    "    \n",
    "\n",
    "def read_experiment_data(experiement_dir, experiment_pattern):\n",
    "    experiemnt_folder_list = get_experiment_folder_list(experiement_dir, experiment_pattern)\n",
    "    experiment_data_tst = {}\n",
    "    experiment_data_val = {}\n",
    "    index_max_tst = 0\n",
    "    index_max_val = 0\n",
    "    \n",
    "    i=0    \n",
    "    for experiement in experiemnt_folder_list:\n",
    "        print(experiement)\n",
    "        print(glob.glob1(experiement+'/eval/', 'events.out.tfevents.*'))\n",
    "        df_tst=[]\n",
    "        df_val=[]\n",
    "        for file in glob.glob1(experiement+'/eval/', 'events.out.tfevents.*'):\n",
    "            ea = EventAccumulator(experiement+'/eval/'+file)\n",
    "            ea.Reload()\n",
    "            tags = ea.Tags()['scalars']\n",
    "#             print(tags)\n",
    "            if not 'accuracy_target_tst' in tags:\n",
    "                continue\n",
    "            data = ea.Scalars('accuracy_target_tst')\n",
    "            if len(data) > 0:\n",
    "                df_tst.append(pd.DataFrame(data).set_index('step').drop('wall_time', axis=1))\n",
    "            data = ea.Scalars('accuracy_target_val')\n",
    "            if len(data) > 0:\n",
    "                df_val.append(pd.DataFrame(data).set_index('step').drop('wall_time', axis=1))\n",
    "        \n",
    "        if len(df_tst) > 0:\n",
    "            df_tst = pd.concat(df_tst)\n",
    "\n",
    "        if len(df_val) > 0:\n",
    "            df_val = pd.concat(df_val)\n",
    "        \n",
    "        if len(df_tst) > 1:\n",
    "            experiment_data_tst[experiement+file] = df_tst\n",
    "            index_max_tst = max(index_max_tst, df_tst.index.max())\n",
    "        \n",
    "        if len(df_tst) > 1:\n",
    "            experiment_data_val[experiement+file] = df_val\n",
    "            index_max_val = max(index_max_val, df_val.index.max())\n",
    "        \n",
    "    return interpolate_data(experiment_data_tst, index_max_tst, step=250), interpolate_data(experiment_data_val, index_max_val, step=250)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "list_experiemnts_target_transfer = get_experiment_folder_list(ROOT_DIR, experiement_patterns[0])\n",
    "list_experiemnts_target_transfer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_data = {}\n",
    "for experiment_pattern in experiement_patterns:\n",
    "    print('Loading experiment: %s' %experiment_pattern)\n",
    "    acc_tst, acc_val = read_experiment_data(ROOT_DIR, experiment_pattern)\n",
    "    \n",
    "    plot_data[(experiment_pattern, 'acc_tst')] = acc_tst\n",
    "    plot_data[(experiment_pattern, 'acc_val')] = acc_val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This is to detect runs that did not converge\n",
    "VALID_RUN_THRESHOLD=0.0\n",
    "\n",
    "for experiment_pattern in experiement_patterns:\n",
    "    print('Loading experiment: %s' %experiment_pattern)\n",
    "    \n",
    "    acc_tst = plot_data[(experiment_pattern, 'acc_tst')]\n",
    "    acc_val = plot_data[(experiment_pattern, 'acc_val')]\n",
    "    \n",
    "#     acc_tst = acc_tst[acc_tst.index<=25000]\n",
    "#     acc_val = acc_val[acc_val.index<=25000]\n",
    "\n",
    "#     columns = list(acc_tst.columns[0:2]) + list(acc_tst.columns[2:-1])\n",
    "#     acc_tst = acc_tst[columns]\n",
    "#     acc_val = acc_val[columns]\n",
    "    \n",
    "    valid_cols = acc_val.max().index[acc_val.max() > VALID_RUN_THRESHOLD]\n",
    "    acc_val = acc_val[valid_cols]\n",
    "    acc_tst = acc_tst[valid_cols]\n",
    "    \n",
    "    acc_tst_ci = sms.DescrStatsW(acc_tst.T).tconfint_mean()\n",
    "    acc_val_ci = sms.DescrStatsW(acc_val.T).tconfint_mean()\n",
    "    acc_val_mean = acc_val.mean(axis=1)\n",
    "    acc_tst_mean = acc_tst.mean(axis=1)\n",
    "    \n",
    "    plot_data[(experiment_pattern, 'acc_tst_mean')] = acc_tst_mean\n",
    "    plot_data[(experiment_pattern, 'acc_val_mean')] = acc_val_mean\n",
    "    plot_data[(experiment_pattern, 'acc_tst_ci')] = acc_tst_ci\n",
    "    plot_data[(experiment_pattern, 'acc_val_ci')] = acc_val_ci\n",
    "    plot_data[(experiment_pattern, 'acc_tst')] = acc_tst\n",
    "    plot_data[(experiment_pattern, 'acc_val')] = acc_val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def argmax_to_max(arr, argmax, axis):\n",
    "    \"\"\"argmax_to_max(arr, arr.argmax(axis), axis) == arr.max(axis)\"\"\"\n",
    "    new_shape = list(arr.shape)\n",
    "    del new_shape[axis]\n",
    "\n",
    "    grid = np.ogrid[tuple(map(slice, new_shape))]\n",
    "    grid.insert(axis, argmax)\n",
    "\n",
    "    return arr[tuple(grid)]\n",
    "\n",
    "\n",
    "def get_max_over_val_bootstrapped(acc_tst, acc_val, bootstrap_ratio, n_mc_trials, replace):\n",
    "    acc_tst = acc_tst.values\n",
    "    acc_val = acc_val.values\n",
    "    assert acc_val.shape[1] == acc_tst.shape[1]\n",
    "    output = np.zeros(shape=(acc_tst.shape[0], n_mc_trials))\n",
    "    for i in range(n_mc_trials):\n",
    "        col_dxs = np.random.choice(range(acc_val.shape[1]), size=round(bootstrap_ratio*acc_val.shape[1]), \n",
    "                                  replace=replace)\n",
    "        acc_val_boot = acc_val[:,col_dxs]\n",
    "        acc_tst_boot = acc_tst[:,col_dxs]\n",
    "                \n",
    "        val_max_idx = np.argmax(acc_val_boot, axis=1)\n",
    "        output[:, i] = argmax_to_max(acc_tst_boot, val_max_idx, axis=1)\n",
    "\n",
    "    return output\n",
    "        \n",
    "    \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "def get_early_stop_over_val(acc_tst, acc_val):\n",
    "    acc_tst = np.nan_to_num(acc_tst.values)\n",
    "    acc_val = np.nan_to_num(acc_val.values)\n",
    "    assert acc_val.shape[1] == acc_tst.shape[1]\n",
    "        \n",
    "    val_max_idx = np.argmax(acc_val, axis=0)\n",
    "    acc_tst_max = argmax_to_max(acc_tst, val_max_idx, axis=0)\n",
    "    \n",
    "    return acc_tst_max\n",
    "\n",
    "print(\"Experiment root folder: %s\" %ROOT_DIR)\n",
    "for i, experiment_pattern in enumerate(experiement_patterns):\n",
    "    acc_tst, acc_val = plot_data[(experiment_pattern, 'acc_tst')], plot_data[(experiment_pattern, 'acc_val')]\n",
    "    acc_tst_validated = get_early_stop_over_val(acc_tst, acc_val)\n",
    "    \n",
    "    acc_tst_validated_ci = sms.DescrStatsW(acc_tst_validated.T).tconfint_mean()\n",
    "    acc_tst_validated_mean = acc_tst_validated.mean()\n",
    "    \n",
    "    print('Experiment name: %s' %(experiment_pattern))\n",
    "    print('mean accuracy: %.1f +/- %.1f' %(100.0*acc_tst_validated_mean, \n",
    "            100.0*(acc_tst_validated_ci[1]-acc_tst_validated_mean)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Experiment root folder: %s\" %ROOT_DIR)\n",
    "for i, experiment_pattern in enumerate(experiement_patterns):\n",
    "    acc_tst, acc_val = plot_data[(experiment_pattern, 'acc_tst')], plot_data[(experiment_pattern, 'acc_val')]\n",
    "    acc_tst_validated = get_early_stop_over_val(acc_val, acc_val)\n",
    "    \n",
    "    acc_tst_validated_ci = sms.DescrStatsW(acc_tst_validated.T).tconfint_mean()\n",
    "    acc_tst_validated_mean = acc_tst_validated.mean()\n",
    "    \n",
    "    print('Experiment name: %s' %(experiment_pattern))\n",
    "    print('mean accuracy: %.1f +/- %.1f' %(100.0*acc_tst_validated_mean, \n",
    "            100.0*(acc_tst_validated_ci[1]-acc_tst_validated_mean)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mplt.rcParams.update({'font.size': 22})\n",
    "\n",
    "majorLocator = MultipleLocator(1.0)\n",
    "majorFormatter = FormatStrFormatter('%.1f')\n",
    "colors=[colorsys.hsv_to_rgb(x*1.0/2.0, 1.0, 1.0) for x in range(2)]\n",
    "\n",
    "fig = plt.figure(figsize=(20,5))\n",
    "ax1 = fig.add_subplot(1,1,1)\n",
    "plt.grid('on')\n",
    "\n",
    "acc_val_vs_degree_mean = np.zeros(len(x_axis_vals))\n",
    "acc_val_vs_degree_ci = np.zeros((len(x_axis_vals), 2))\n",
    "acc_tst_vs_degree_mean = np.zeros(len(x_axis_vals))\n",
    "acc_tst_vs_degree_ci = np.zeros((len(x_axis_vals), 2))\n",
    "for i, experiment_pattern in enumerate(experiement_patterns):\n",
    "    acc_tst, acc_val = plot_data[(experiment_pattern, 'acc_tst')], plot_data[(experiment_pattern, 'acc_val')]\n",
    "    \n",
    "    # This is based on early stop over each run\n",
    "    acc_tst_validated = get_early_stop_over_val(acc_tst, acc_val)\n",
    "    acc_val_validated = get_early_stop_over_val(acc_val, acc_val)\n",
    "    \n",
    "#     # This is based on early stop over the mean\n",
    "#     acc_val_mean = acc_val.mean(axis=1)\n",
    "#     stop_idx = acc_val_mean.argmax()\n",
    "#     acc_val_validated = acc_val.loc[stop_idx]\n",
    "#     acc_tst_validated = acc_tst.loc[stop_idx]\n",
    "    \n",
    "    acc_val_vs_degree_ci[i] = sms.DescrStatsW(acc_val_validated.T).tconfint_mean()\n",
    "    acc_val_vs_degree_mean[i] = acc_val_validated.mean()\n",
    "    \n",
    "    acc_tst_vs_degree_ci[i] = sms.DescrStatsW(acc_tst_validated.T).tconfint_mean()\n",
    "    acc_tst_vs_degree_mean[i] = acc_tst_validated.mean()\n",
    "    \n",
    "    \n",
    "ax1.plot(x_axis_vals, 100*acc_val_vs_degree_mean, label='validation', c=colors[0])\n",
    "ax1.fill_between(x_axis_vals, 100.0*acc_val_vs_degree_ci[:,0], 100.0*acc_val_vs_degree_ci[:,1], \n",
    "                alpha=0.25, edgecolor=colors[0], facecolor=colors[0])\n",
    "\n",
    "ax1.plot(x_axis_vals, 100*acc_tst_vs_degree_mean, label='test', c=colors[1])\n",
    "ax1.fill_between(x_axis_vals, 100.0*acc_tst_vs_degree_ci[:,0], 100.0*acc_tst_vs_degree_ci[:,1], \n",
    "                alpha=0.25, edgecolor=colors[1], facecolor=colors[1])\n",
    "\n",
    "ylim_min = 100*min(acc_val_vs_degree_mean.min(), acc_tst_vs_degree_mean.min()) - 0.5\n",
    "ylim_max = 100*max(acc_val_vs_degree_ci.max(), acc_tst_vs_degree_ci.max()) + 0.5\n",
    "\n",
    "legend = ax1.legend(shadow=True, loc=4)\n",
    "ax1.grid(which='minor', alpha=0.5)                                                \n",
    "ax1.grid(which='major', alpha=1.0) \n",
    "ax1.set_ylim([ylim_min,ylim_max])\n",
    "ax1.set_xlim([0.0,max(x_axis_vals)+0.5])\n",
    "\n",
    "ax1.set_xlabel(r'Scale, $\\alpha$')\n",
    "ax1.set_xticks(range(1, math.floor(max(x_axis_vals))+1))\n",
    "ax1.set_ylabel('Accuracy, %')\n",
    "ax1.yaxis.set_major_locator(majorLocator)\n",
    "ax1.yaxis.set_major_formatter(majorFormatter)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "acc_val_vs_degree_mean*100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "acc_tst_vs_degree_mean*100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
